package com.cybertron.ironhide.spider;

import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Map;

import org.eweb4j.util.CommonUtil;

import com.cybertron.ironhide.spider.domain.TaskUrl;
import com.cybertron.ironhide.spider.fetcher.FetchResult;
import com.cybertron.ironhide.spider.fetcher.Page;
import com.cybertron.ironhide.spider.plugin.DoneException;
import com.cybertron.ironhide.spider.task.Task;
import com.cybertron.ironhide.spider.xml.Field;
import com.cybertron.ironhide.spider.xml.Site;
import com.cybertron.ironhide.spider.xml.Target;

/**
 * 网络蜘蛛
 * 
 * @author weiwei
 * 
 */
public class Spider implements Runnable {

	public Task task;

	private Site site;

	public SpiderListener listener;

	public Spider(Task task, Site site, SpiderListener listener) {
		this.task = task;
		this.listener = listener;
		this.site = site;
	}

	public void run() {
		try {
			// 扩展点：begin 蜘蛛开始
			if (site.beginPoint != null) {
				task = site.beginPoint.confirmTask(task);
			}
			if (task == null)
				return;

			if (task.site.isStop)
				return;

			// 扩展点：fetch 获取HTTP内容

			FetchResult result = null;
			if (site.fetchPoint != null) {
				result = site.fetchPoint.fetch(task, result);
			}
			listener.onFetch(Thread.currentThread(), task, result);

			if (result == null)
				return;

			if (task.site.isStop)
				return;

			// 扩展点：dig new url 发觉新URL
			List<TaskUrl> newUrls = site.digPoint.digNewUrls(result, task);

			if (task.site.isStop)
				return;

			handleNewUrls(newUrls);

			if (task.site.isStop)
				return;

			Page page = result.getPage();
			if (page == null) {
				return;
			}

			if (task.site.isStop)
				return;

			// 扩展点：target 确认是否有目标配置匹配当前URL
			Target target = null;
			if (site.targetPoint != null) {
				target = site.targetPoint.confirmTarget(task);
			}

			if (target == null) {
				return;
			}

			task.target = target;
			this.listener.onTargetPage(Thread.currentThread(), task, page);

			if (task.site.isStop)
				return;

			// 扩展点：parse 把已确认好的目标页面解析成为Map对象
			List<Map<String, Object>> models = null;
			if (site.parsePoint != null) {
				models = site.parsePoint.parse(task, target, page);
			}

			if (models == null) {
				return;
			}

			for (Iterator<Map<String, Object>> _it = models.iterator(); _it
					.hasNext();) {
				Map<String, Object> model = _it.next();
				for (Iterator<Field> it = target.getModel().getField()
						.iterator(); it.hasNext();) {
					Field f = it.next();
					// 去掉那些被定义成 参数 的field
					if ("1".equals(f.getIsParam())
							|| "true".equals(f.getIsParam()))
						model.remove(f.getName());
				}
				model.put("source_url", task.sourceUrl);
				model.put("task_url", task.url);
			}

			// 统计任务完成数+1
			this.task.site.counter.plus();

			listener.onParse(Thread.currentThread(), task, models);

			if (task.digNewUrls != null && !task.digNewUrls.isEmpty()) {
				List<TaskUrl> taskUrls = task.digNewUrls;
				if (!taskUrls.isEmpty()) {
					handleNewUrls(taskUrls);
					task.digNewUrls.clear();
					task.digNewUrls = null;
				}
			}

			listener.onInfo(Thread.currentThread(), task, "site -> "
					+ task.site.getName() + " task parse finished count ->"
					+ task.site.counter.getCount());

			if (task.site.isStop)
				return;

			// 扩展点：pojo 将Map数据映射为POJO
			String modelCls = target.getModel().getClazz();
			Class<?> cls = null;
			if (modelCls != null)
				cls = Thread.currentThread().getContextClassLoader()
						.loadClass(modelCls);

			List<Object> pojos = null;
			if (site.pojoPoint != null) {
				pojos = site.pojoPoint.mapping(task, cls, models);
			}

			if (pojos != null)
				listener.onPojo(Thread.currentThread(), task, pojos);

			if (task.site.isStop)
				return;

			// 扩展点：end 蜘蛛完成工作，该收尾了
			if (site.endPoint != null) {
				models = site.endPoint.complete(task, models);
			}

		} catch (DoneException e) {
			if (this.listener != null)
				this.listener.onInfo(Thread.currentThread(), task,
						"Spiderman has shutdown already...");
		} catch (Throwable e) {
			if (this.listener != null)
				this.listener.onError(Thread.currentThread(), task,
						CommonUtil.getExceptionString(e), e);
		}
	}

	private void handleNewUrls(List<TaskUrl> newUrls) throws Exception {
		if (newUrls != null && !newUrls.isEmpty()) {
			this.listener.onNewUrls(Thread.currentThread(), task, newUrls);
		} else {
			newUrls = new ArrayList<TaskUrl>();
		}

		if (task.site.isStop)
			return;

		// 扩展点：dup_removal URL去重,然后变成Task
		List<Task> validTasks = site.dupRemovalPoint.removeDuplicateTask(task,
				newUrls);

		if (newUrls != null && !newUrls.isEmpty()) {
			this.listener
					.onDupRemoval(Thread.currentThread(), task, validTasks);
		}

		if (validTasks == null)
			validTasks = new ArrayList<Task>();

		if (task.site.isStop)
			return;

		// 扩展点：task_sort 给任务排序
		validTasks = site.taskPoint.sortTasks(validTasks);

		this.listener.onTaskSort(Thread.currentThread(), task, validTasks);

		if (validTasks == null)
			validTasks = new ArrayList<Task>();

		if (task.site.isStop)
			return;

		// 扩展点：task_push 将任务放入队列
		validTasks = pushTask(validTasks);
		if (validTasks != null && !validTasks.isEmpty())
			this.listener.onNewTasks(Thread.currentThread(), task, validTasks);
	}

	public List<Task> pushTask(List<Task> validTasks) throws Exception {
		if (site.taskPoint != null) {
			site.taskPoint.pushTask(validTasks);
		}
		return validTasks;
	}
}
